{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:38.117662Z",
     "iopub.status.busy": "2022-09-19T02:13:38.117076Z",
     "iopub.status.idle": "2022-09-19T02:13:38.196949Z",
     "shell.execute_reply": "2022-09-19T02:13:38.196268Z",
     "shell.execute_reply.started": "2022-09-19T02:13:38.117607Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pandas 数据集读取，dataframe形式的\n",
    "import pandas as pd\n",
    "import codecs\n",
    "\n",
    "train_df = pd.read_csv('./中文对话文本匹配挑战赛数据集/train.csv', sep='\\t', header=None, nrows=None)\n",
    "test_df = pd.read_csv('./中文对话文本匹配挑战赛数据集/test.csv', sep='\\t', header=None)\n",
    "\n",
    "# train_df = train_df[train_df['question2'].apply(lambda x: isinstance(x, str))]\n",
    "# train_df = train_df[train_df['question1'].apply(lambda x: isinstance(x, str))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:38.397765Z",
     "iopub.status.busy": "2022-09-19T02:13:38.397209Z",
     "iopub.status.idle": "2022-09-19T02:13:38.405685Z",
     "shell.execute_reply": "2022-09-19T02:13:38.405145Z",
     "shell.execute_reply.started": "2022-09-19T02:13:38.397716Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>藏獒为什么这么贵</td>\n",
       "      <td>藏獒见人不咬为什么</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>人生应该怎么才算精彩？</td>\n",
       "      <td>人生要怎么过才算精彩啊</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>为什么打牌老是输</td>\n",
       "      <td>为什么我枪神纪进不去了</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>现在网上卖什么最赚钱</td>\n",
       "      <td>网上卖什么最赚钱</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>如何提高气质</td>\n",
       "      <td>怎样提高自身气质？</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             0            1  2\n",
       "0     藏獒为什么这么贵    藏獒见人不咬为什么  0\n",
       "1  人生应该怎么才算精彩？  人生要怎么过才算精彩啊  1\n",
       "2     为什么打牌老是输  为什么我枪神纪进不去了  0\n",
       "3   现在网上卖什么最赚钱     网上卖什么最赚钱  1\n",
       "4       如何提高气质    怎样提高自身气质？  1"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:38.597100Z",
     "iopub.status.busy": "2022-09-19T02:13:38.596830Z",
     "iopub.status.idle": "2022-09-19T02:13:38.602874Z",
     "shell.execute_reply": "2022-09-19T02:13:38.602436Z",
     "shell.execute_reply.started": "2022-09-19T02:13:38.597079Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>成语中的历史人物</td>\n",
       "      <td>成语有关的历史人物</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>黄财神怎样供奉</td>\n",
       "      <td>怎样供奉财神</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>进门是餐厅好吗</td>\n",
       "      <td>进门见餐厅好吗</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>怎么提高理解力？</td>\n",
       "      <td>怎样提高理解力</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>仓鼠用什么磨牙都可以吗</td>\n",
       "      <td>用什么可以代替仓鼠的磨牙棒</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             0              1\n",
       "0     成语中的历史人物      成语有关的历史人物\n",
       "1      黄财神怎样供奉         怎样供奉财神\n",
       "2      进门是餐厅好吗        进门见餐厅好吗\n",
       "3     怎么提高理解力？        怎样提高理解力\n",
       "4  仓鼠用什么磨牙都可以吗  用什么可以代替仓鼠的磨牙棒"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:38.777737Z",
     "iopub.status.busy": "2022-09-19T02:13:38.777186Z",
     "iopub.status.idle": "2022-09-19T02:13:38.785328Z",
     "shell.execute_reply": "2022-09-19T02:13:38.784731Z",
     "shell.execute_reply.started": "2022-09-19T02:13:38.777687Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    29039\n",
       "0    20961\n",
       "Name: 2, dtype: int64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[2].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:39.003448Z",
     "iopub.status.busy": "2022-09-19T02:13:39.002841Z",
     "iopub.status.idle": "2022-09-19T02:13:39.009373Z",
     "shell.execute_reply": "2022-09-19T02:13:39.008471Z",
     "shell.execute_reply.started": "2022-09-19T02:13:39.003406Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 3)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:08.358976Z",
     "start_time": "2021-03-11T09:35:08.347065Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:39.247066Z",
     "iopub.status.busy": "2022-09-19T02:13:39.246529Z",
     "iopub.status.idle": "2022-09-19T02:13:39.253239Z",
     "shell.execute_reply": "2022-09-19T02:13:39.252038Z",
     "shell.execute_reply.started": "2022-09-19T02:13:39.247019Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:09.263768Z",
     "start_time": "2021-03-11T09:35:09.227357Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:39.567390Z",
     "iopub.status.busy": "2022-09-19T02:13:39.566844Z",
     "iopub.status.idle": "2022-09-19T02:13:39.597309Z",
     "shell.execute_reply": "2022-09-19T02:13:39.596733Z",
     "shell.execute_reply.started": "2022-09-19T02:13:39.567341Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 划分为训练集和验证集\n",
    "# stratify 按照标签进行采样，训练集和验证部分同分布\n",
    "q1_train, q1_val, q2_train, q2_val, train_label, val_label = train_test_split(\n",
    "    train_df[0].iloc[:],\n",
    "    train_df[1].iloc[:],\n",
    "    train_df[2].iloc[:],\n",
    "    test_size=0.1,\n",
    "    stratify=train_df[2].iloc[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:39.997916Z",
     "iopub.status.busy": "2022-09-19T02:13:39.997320Z",
     "iopub.status.idle": "2022-09-19T02:13:40.001459Z",
     "shell.execute_reply": "2022-09-19T02:13:40.000874Z",
     "shell.execute_reply.started": "2022-09-19T02:13:39.997863Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# input_ids：字的编码\n",
    "# token_type_ids：标识是第一个句子还是第二个句子\n",
    "# attention_mask：标识是不是填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:19.935035Z",
     "start_time": "2021-03-11T09:35:09.908638Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:40.201888Z",
     "iopub.status.busy": "2022-09-19T02:13:40.201364Z",
     "iopub.status.idle": "2022-09-19T02:13:56.034242Z",
     "shell.execute_reply": "2022-09-19T02:13:56.033633Z",
     "shell.execute_reply.started": "2022-09-19T02:13:40.201839Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pip install transformers\n",
    "# transformers bert相关的模型使用和加载\n",
    "from transformers import BertTokenizer\n",
    "# 分词器，词典\n",
    "import transformers\n",
    "transformers.logging.set_verbosity_error()\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "train_encoding = tokenizer(list(q1_train), list(q2_train),\n",
    "                           truncation=True, padding=True, max_length=32)\n",
    "\n",
    "val_encoding = tokenizer(list(q1_val), list(q2_val),\n",
    "                          truncation=True, padding=True, max_length=32)\n",
    "\n",
    "test_encoding = tokenizer(list(test_df[0]), list(test_df[1]),\n",
    "                          truncation=True, padding=True, max_length=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:43.578135Z",
     "start_time": "2021-03-11T09:35:43.571452Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:56.035525Z",
     "iopub.status.busy": "2022-09-19T02:13:56.035268Z",
     "iopub.status.idle": "2022-09-19T02:13:56.042099Z",
     "shell.execute_reply": "2022-09-19T02:13:56.041600Z",
     "shell.execute_reply.started": "2022-09-19T02:13:56.035507Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 数据集读取\n",
    "class QuoraDataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    # 读取单个样本\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(int(self.labels[idx]))\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "\n",
    "train_dataset = QuoraDataset(train_encoding, list(train_label))\n",
    "val_dataset = QuoraDataset(val_encoding, list(val_label))\n",
    "test_dataset = QuoraDataset(test_encoding, [0] * len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:44.110121Z",
     "start_time": "2021-03-11T09:35:44.104871Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:56.043059Z",
     "iopub.status.busy": "2022-09-19T02:13:56.042897Z",
     "iopub.status.idle": "2022-09-19T02:13:56.110234Z",
     "shell.execute_reply": "2022-09-19T02:13:56.109150Z",
     "shell.execute_reply.started": "2022-09-19T02:13:56.043044Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 精度计算\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:58:49.027161Z",
     "start_time": "2021-03-11T09:58:45.317009Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:56.112496Z",
     "iopub.status.busy": "2022-09-19T02:13:56.112017Z",
     "iopub.status.idle": "2022-09-19T02:13:58.850368Z",
     "shell.execute_reply": "2022-09-19T02:13:58.849837Z",
     "shell.execute_reply.started": "2022-09-19T02:13:56.112451Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lyz/.local/lib/python3.6/site-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForNextSentencePrediction, AdamW, get_linear_schedule_with_warmup\n",
    "model = BertForNextSentencePrediction.from_pretrained('bert-base-chinese')\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "# 单个读取到批量读取\n",
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
    "val_dataloader = DataLoader(val_dataset, batch_size=16, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=False)\n",
    "\n",
    "# 优化方法\n",
    "optim = AdamW(model.parameters(), lr=2e-5)\n",
    "total_steps = len(train_loader) * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:44:22.077501Z",
     "start_time": "2021-03-11T09:39:16.473609Z"
    },
    "execution": {
     "iopub.execute_input": "2022-09-19T02:13:58.851413Z",
     "iopub.status.busy": "2022-09-19T02:13:58.851242Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 100, loss: 0.0928, 3.55%\n",
      "epoth: 0, iter_num: 200, loss: 0.4208, 7.11%\n",
      "epoth: 0, iter_num: 300, loss: 0.1421, 10.66%\n",
      "epoth: 0, iter_num: 400, loss: 0.3197, 14.22%\n",
      "epoth: 0, iter_num: 500, loss: 0.2212, 17.77%\n",
      "epoth: 0, iter_num: 600, loss: 0.3806, 21.33%\n",
      "epoth: 0, iter_num: 700, loss: 0.3617, 24.88%\n",
      "epoth: 0, iter_num: 800, loss: 0.1321, 28.44%\n",
      "epoth: 0, iter_num: 900, loss: 0.6039, 31.99%\n",
      "epoth: 0, iter_num: 1000, loss: 0.1054, 35.55%\n",
      "epoth: 0, iter_num: 1100, loss: 0.3961, 39.10%\n",
      "epoth: 0, iter_num: 1200, loss: 0.3404, 42.66%\n",
      "epoth: 0, iter_num: 1300, loss: 0.4050, 46.21%\n",
      "epoth: 0, iter_num: 1400, loss: 0.5042, 49.77%\n",
      "epoth: 0, iter_num: 1500, loss: 0.1580, 53.32%\n",
      "epoth: 0, iter_num: 1600, loss: 0.4447, 56.88%\n",
      "epoth: 0, iter_num: 1700, loss: 0.1941, 60.43%\n",
      "epoth: 0, iter_num: 1800, loss: 0.2821, 63.99%\n",
      "epoth: 0, iter_num: 1900, loss: 0.3955, 67.54%\n",
      "epoth: 0, iter_num: 2000, loss: 0.2193, 71.10%\n",
      "epoth: 0, iter_num: 2100, loss: 0.4951, 74.65%\n",
      "epoth: 0, iter_num: 2200, loss: 0.1926, 78.21%\n",
      "epoth: 0, iter_num: 2300, loss: 0.4419, 81.76%\n",
      "epoth: 0, iter_num: 2400, loss: 0.2570, 85.32%\n",
      "epoth: 0, iter_num: 2500, loss: 0.2686, 88.87%\n",
      "epoth: 0, iter_num: 2600, loss: 0.3637, 92.43%\n",
      "epoth: 0, iter_num: 2700, loss: 0.3477, 95.98%\n",
      "epoth: 0, iter_num: 2800, loss: 0.4032, 99.54%\n",
      "Epoch: 0, Average training loss: 0.3003\n",
      "Accuracy: 0.8946\n",
      "Average validation loss: 0.2499\n",
      "------------Epoch: 1 ----------------\n",
      "epoth: 1, iter_num: 100, loss: 0.2685, 3.55%\n",
      "epoth: 1, iter_num: 200, loss: 0.0915, 7.11%\n",
      "epoth: 1, iter_num: 300, loss: 0.0136, 10.66%\n",
      "epoth: 1, iter_num: 400, loss: 0.5172, 14.22%\n",
      "epoth: 1, iter_num: 500, loss: 0.0786, 17.77%\n",
      "epoth: 1, iter_num: 600, loss: 0.0888, 21.33%\n",
      "epoth: 1, iter_num: 700, loss: 0.6134, 24.88%\n",
      "epoth: 1, iter_num: 800, loss: 0.0968, 28.44%\n",
      "epoth: 1, iter_num: 900, loss: 0.0947, 31.99%\n",
      "epoth: 1, iter_num: 1000, loss: 0.0596, 35.55%\n",
      "epoth: 1, iter_num: 1100, loss: 0.1687, 39.10%\n",
      "epoth: 1, iter_num: 1200, loss: 0.1477, 42.66%\n",
      "epoth: 1, iter_num: 1300, loss: 0.0246, 46.21%\n",
      "epoth: 1, iter_num: 1400, loss: 0.2130, 49.77%\n",
      "epoth: 1, iter_num: 1500, loss: 0.1065, 53.32%\n",
      "epoth: 1, iter_num: 1600, loss: 0.2308, 56.88%\n",
      "epoth: 1, iter_num: 1700, loss: 0.0357, 60.43%\n",
      "epoth: 1, iter_num: 1800, loss: 0.3935, 63.99%\n",
      "epoth: 1, iter_num: 1900, loss: 0.2334, 67.54%\n",
      "epoth: 1, iter_num: 2000, loss: 0.2451, 71.10%\n",
      "epoth: 1, iter_num: 2100, loss: 0.0234, 74.65%\n",
      "epoth: 1, iter_num: 2200, loss: 0.1034, 78.21%\n",
      "epoth: 1, iter_num: 2300, loss: 0.3372, 81.76%\n",
      "epoth: 1, iter_num: 2400, loss: 0.0815, 85.32%\n",
      "epoth: 1, iter_num: 2500, loss: 0.1564, 88.87%\n"
     ]
    }
   ],
   "source": [
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "\n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 100 == 0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "\n",
    "    print(\"Epoch: %d, Average training loss: %.4f\" % (epoch, total_train_loss/len(train_loader)))\n",
    "\n",
    "\n",
    "def validation():\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    for batch in val_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "\n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "\n",
    "        total_eval_loss += loss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = labels.to('cpu').numpy()\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "\n",
    "    avg_val_accuracy = total_eval_accuracy / len(val_dataloader)\n",
    "    print(\"Accuracy: %.4f\" % (avg_val_accuracy))\n",
    "    print(\"Average validation loss: %.4f\" % (total_eval_loss/len(val_dataloader)))\n",
    "\n",
    "\n",
    "for epoch in range(3):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()\n",
    "    validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:12:11.722405Z",
     "iopub.status.busy": "2022-09-19T02:12:11.721805Z",
     "iopub.status.idle": "2022-09-19T02:12:11.729014Z",
     "shell.execute_reply": "2022-09-19T02:12:11.728217Z",
     "shell.execute_reply.started": "2022-09-19T02:12:11.722352Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def prediciton():\n",
    "    model.eval()\n",
    "    prediction_list = []\n",
    "    for batch in test_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "\n",
    "        logits = outputs[1]\n",
    "        \n",
    "        preds = logits.detach().cpu().numpy()\n",
    "        pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "        prediction_list += list(pred_flat)\n",
    "    return prediction_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:12:12.362677Z",
     "iopub.status.busy": "2022-09-19T02:12:12.362112Z",
     "iopub.status.idle": "2022-09-19T02:12:34.197470Z",
     "shell.execute_reply": "2022-09-19T02:12:34.196834Z",
     "shell.execute_reply.started": "2022-09-19T02:12:12.362625Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_result = prediciton()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-19T02:12:42.335236Z",
     "iopub.status.busy": "2022-09-19T02:12:42.334621Z",
     "iopub.status.idle": "2022-09-19T02:12:42.360138Z",
     "shell.execute_reply": "2022-09-19T02:12:42.359442Z",
     "shell.execute_reply.started": "2022-09-19T02:12:42.335183Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(test_result).to_csv('submit.csv', header=None, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
